{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM-CNN Filter Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Setup.ipynb\n",
    "%run ExtraFunctions.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load embedding matrix into an `Embedding` layer. Toggle `trainable=False` to prevent the weights from being updated during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN Structure\n",
    "[Reference](https://github.com/richliao/textClassifier), [LTSM](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "l_lstm1 = Bidirectional(LSTM(4,dropout=0.2,recurrent_dropout=0.2,return_sequences=True))(embedded_sequences)\n",
    "l_cov1= Conv1D(24, 9, activation='relu')(l_lstm1)\n",
    "l_pool1 = MaxPooling1D(2)(l_cov1)\n",
    "l_drop1 = Dropout(0.3)(l_pool1)\n",
    "l_flat = Flatten()(l_drop1)\n",
    "l_dense = Dense(16, activation='relu')(l_flat)\n",
    "preds = Dense(4, activation='softmax')(l_dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = Model(sequence_input, preds)\n",
    "adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "lr_metric = get_lr_metric(adadelta)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=adadelta,\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=16, write_grads=True , write_graph=True)\n",
    "model_checkpoints = callbacks.ModelCheckpoint(\"checkpoint.h5\", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)\n",
    "lr_schedule = callbacks.LearningRateScheduler(initial_boost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()\n",
    "print(\"Training Progress:\")\n",
    "history = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "                    epochs=50, batch_size=64,\n",
    "                    callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(history.history).to_csv(\"history-9.csv\")\n",
    "#model.save('ltsm-c-3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
